package com.hh.test.crawler;

import java.io.File;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.hh.test.util.FileUtil;
import com.hh.test.util.HttpUtil;

public class Crawler {

	private int sum = 0;

	static String encode = "GBK";
	static String htmlencode = "GBK";

	public static void main(String[] args) {

		Map<String, String> map = new HashMap<String, String>();
		// map.put("2017福州市区笔试成绩",
		// "http://218.66.46.49:4002/special/jszk/2017/cjcx/jszk_0506_03_BE9E928EE959E8F7.htm");
		// map.put("2017福州市区面试成绩",
		// "http://218.66.46.49:4002/special/jszk/2017/cjcx/jszk_0506_04_69F7316E5BA74AAE.htm");
//		 map.put("2016福州市区笔试成绩",
//		 "http://www.fzedu.gov.cn/jxgkxx/News/2016/05/25/26711.html");
//		 map.put("2016福州市区面试成绩",
//		 "http://218.66.46.49:4002/special/jszk/2016/cjcx/jszk_lqjg_03.htm");
		 map.put("2015福州市区面试成绩",
		 "http://www.fzedu.gov.cn/jxgkxx/News/2015/06/03/24843.html");
		// map.put("2015福州市区笔试成绩",不能用
		// "http://www.fzedu.gov.cn/jxgkxx/News/2015/05/27/24811.html");

		Iterator<Entry<String, String>> entry = map.entrySet().iterator();
		while (entry.hasNext()) {
			Entry<String, String> en = entry.next();
			Crawler c = new Crawler();
			String dir = Crawler.class.getClassLoader().getResource("")
					.getPath()
					+ en.getKey();
			dir = "I:\\Users\\chf\\Desktop\\cj\\" + en.getKey();
			new File(dir).mkdirs();
			c.snatch(en.getValue(), dir);
		}

	}

	void snatch(String url, String dir) {
		// System.out.println(url);
		// System.out.println(dir);
		String relativePath = "";
		String html = HttpUtil.getResByUrl(url, "UTF-8");
		relativePath = url.substring(0, url.lastIndexOf("/"));
		// System.out.println(relativePath);
		// System.out.println(dir + name);
		// System.out.println(html);
		String name = "page" + sum + ".htm";
		sum++;
		String pattern = "<a href=\"(\\S*?)\">";
		Pattern p = Pattern.compile(pattern,Pattern.CASE_INSENSITIVE);
		Matcher matcher = p.matcher(html);
		while (matcher.find()) {
			String matcherUrl = matcher.group(1);
			html = html.replace(matcherUrl, "page" + sum + ".htm");
			matcherUrl=matcherUrl.replaceAll("&amp;", "&");
			if (matcherUrl.startsWith("http")) {
				inSnatch(matcherUrl, dir);
			} else {
				inSnatch(relativePath + "/" + matcherUrl, dir);
			}
		}
		FileUtil.strToFile(dir + "/" + name, encode, html);
	}

	void inSnatch(String url, String dir) {
		String html = HttpUtil.getResByUrl(url, htmlencode);
		String name = "page" + sum + ".htm";
		sum++;
		FileUtil.strToFile(dir + "/" + name, encode, html);
	}

}
